from lxml import etree
import os
import requests
import parsel
import re




def get_url():
    biquge_url = 'http://www.xbiquge.la'
    url0 = 'http://www.biquge.info/list/1_1.html'
    response = requests.get(url0, headers=headers)
    response.encoding = response.apparent_encoding
    response_text = parsel.Selector(response.text)
    response_url = response_text.xpath('//*[@id="newscontent"]/div[2]/ul//li/span[2]/a/@href').extract()
    # author = response_text.xpath('//*[@id="newscontent"]/div[2]/ul//li/span[2]/text()').extract()
    title = response_text.xpath('//*[@id="newscontent"]/div[2]/ul//li/span[2]/a/text()').extract()
    for p, q in zip(title, response_url[0:-1]):
        os.mkdir(r'D:\桌面\小说\ ' + p)
        req = requests.get(q, headers=headers)
        req.encoding = req.apparent_encoding
        req1 = req.text
        html = parsel.Selector(req1)
        title_name = html.xpath('//*[@id="list"]/dl/dd/a/text()').extract()
        print(title_name)
        mulv = html.xpath('//*[@id="list"]/dl/dd/a/@href').extract()
        print(mulv)
        mvlv_url = [q + x for x in mulv]
        for a, y in zip(mvlv_url, title_name):
            print(a, y)
            rstr = r'[\/\\\:\*\?\"\<\>\|]'
            new_title = re.sub(rstr, "_", y)
            session = requests.Session()
            r = session.get(a, headers=headers)
            r.encoding = r.apparent_encoding
            # print(r.text)
            selector = etree.HTML(r.text)
            print(2)
            # 获取文章标题
            # 获取文本内容
            text = selector.xpath('//*[@id="content"]/text()')
            path = r'D:\桌面\小说\ ' + p + '\ '
            with open(path + new_title + '.text', 'w', encoding='utf-8') as f:
                for i in text:
                    f.write(i+'\n'+'\n')

if __name__ == '__main__':
    headers = {
        'Referer': 'http://www.23us.com/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36'
    }
    get_url()